#!/usr/bin/python2.4
#
# Copyright 2006 - 2008 Google Inc. All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

"""Yet Another Windows Tester."""

import StringIO
import binascii
import codecs
import copy
import getopt
import glob
import inspect
import locale
import logging
import os
import random
import re
import sha
import shutil
import string
import subprocess
import sys
import tempfile
import time
import urllib
from types import *

########################################################################
#
# YAWT Configuration
#
########################################################################

# These are the default configuration values, they are required unless
# specified otherwise.

_yawt_default_configuration = {

# Common options to all platforms
# (can be overwritten on a given platform by specifying the option in the OS
# specific configuration file.)

  'common': {
    'overwrite': 0,
    'dry_run': 0,
    'no_verify': 0,

    # 1 - critical only
    # 2 - error and above
    # 3 - warning and above
    # 4 - info and above
    # 5 - all messages
    'verbosity': 4,

    # Autohotkey is a program for automating testing.
    'autohotkey_bin': 'bin/AutoHotkey.exe',
    'autohotkey_sum': '9b780c73f053dae283ef744c4b6b1a06ef276264',
    'autohotkey_url':
      'http://yawt.googlecode.com/files/AutoHotkey-1.0.46.14.exe',

    # Cabextract is needed to extract .cab files.
    'cabextract_bin': 'bin/cabextract.exe',
    'cabextract_sum': 'e9a16bd41663152e9fd7de3b872d2e36cb657762',
    'cabextract_url': 'http://yawt.googlecode.com/files/cabextract-1.2.exe',

    # Regread is a program to read registry keys.
    'regread_bin': 'bin/regread.exe',
    'regread_sum': 'c0379f725a848dd6dba71331d799b274305306d1',
    'regread_url': 'http://yawt.googlecode.com/files/regread-0.1.exe',

    # Unzip is needed to extract .zip files.
    'unzip_bin': 'bin/unzip.exe',
    'unzip_sum': '528588b7eca3ea65782298cc44d547b7f7722140',
    'unzip_url': 'http://yawt.googlecode.com/files/unzip-5.52.exe',
  },

# Linux specific options. (these will override the common option)

  'linux2': {
    # The following options are specific to platforms that run Wine:

    # The wine directory (wine_dir) contains all the user specific data for
    # wine, including the registry and the c: drive. By default, wine_dir is a
    # randomly directory created by mktemp, so one can run tests repeatedly
    # without cleaning up the wine directory. To override this behavior, one
    # can set the environment variable WINEPREFIX. Alternatively, set the
    # variable below (overrides WINEPREFIX)
    #'wine_dir': '~/.wine', # OPTIONAL

    # The wine executable can be found by searching the PATH
    # uncomment the line below to override the default location
    #'wine_bin': '/path/to/wine', # OPTIONAL

    # The wineprefixcreate executable can be found by searching the PATH
    # uncomment the line below to override the default location
    #'wineprefixcreate_bin': '/path/to/wineprefixcreate', # OPTIONAL

    # The path to the CD drive on Linux - you must include the trailing '/'
    # uncomment the line below if you have tests that use the cd:// URI
    #'cd_dir': '/media/cdrom/', # OPTIONAL
  },

# Win32 (i.e. Activestate Python) specific options (these will override the
# common option)

  'win32': {
    # The path to the CD drive on Windows - you must include the trailing '/'
    # uncomment the line below if you have tests that use the cd:// URI
    #'cd_dir': 'd:/', # OPTIONAL
  },
}

########################################################################
#
# Configuration Classes
#
########################################################################

class BaseConfig:

  """Base container class.

  A lockable container class for storing key/value pairs that can be extended
  to hold different types of configurations. When locked, the container becomes
  read-only, and all writes/modifications result in an exception.
  """

  def __init__(self):
    """Initializes the container as empty and unlocked."""
    self._locked = False
    self._options = {}

  def IsLocked(self):
    """Returns whether the container is locked or not."""
    return self._locked

  def Lock(self):
    """Lock the container, making it read-only."""

    self._locked = True

  def Get(self, key):
    """Gets the value for key.

    Args:
      key: the key to get the value for.

    Returns:
      Value for key
    """
    if key not in self._options:
      raise WineTestConfigError(key)
    return self._options[key]

  def GetAll(self):
    """Get all the key/value pairs.

    Returns:
      Dictionary with all the key/value pairs
    """
    return self._options

  def Set(self, key, value):
    """Sets the value, when overwriting an existing value, only do so if the
    type of the current value and new value matches.

    Args:
      key: the key to set the value for.
      value: the value to set key to.
    """
    if self._locked:
      raise WineTestWriteError(key)
    if key in self._options:
      if type(self._options[key]) != type(value):
        raise WineTestTypeError(key)
    self._options[key] = value

  def Delete(self, key):
    """Delete key from the container. """
    if self._locked:
      raise WineTestWriteError(key)
    del self._options[key]


class GenericConfig(BaseConfig):

  """Generic configuration class for YAWT.

  This is a generic configuration class that holds the basic configuration
  needed for any of the commands YAWT can run. This class performs some
  rudimentary checks on the configuration data, and it has a few useful helper
  methods.
  """

  def __init__(self, base_dir, command_line_options):
    """Initialize the configuration parameters and check them.

    This function sets up the configuration parameters using some sensible
    defaults, as well as configuration data from the 'config' directory and the
    command line.

    Args:
      base_dir: the directory containing YAWT
      command_line_options: list of options from the command line
    """
    BaseConfig.__init__(self)
    self.data_types = {
      'download_dir': StringType,
      'test_dir': StringType,
      'verbosity': IntType,
    }

    if base_dir:
      # let us use '~/path/to/foo'
      base_dir = os.path.expanduser(base_dir)
    else:
      # to make 'python yawt.py' work, we set base_dir to '.' when it's blank
      base_dir = os.path.abspath('.')

    # some built-in parameters that probably don't need to change
    self._options['base_dir'] = base_dir
    self._options['download_dir'] = os.path.join(base_dir, 'downloads')
    self._options['test_dir'] = os.path.join(base_dir, 'tests')

    # load default configuration options
    common_options = _yawt_default_configuration['common']
    os_options = _yawt_default_configuration[sys.platform]

    # merge all the options
    # the order of precedence (least important to most important)
    # common options, os options, command line
    for k, v in common_options.iteritems():
      self._options[k] = v
    for k, v in os_options.iteritems():
      self._options[k] = v
    for k, v in command_line_options.iteritems():
      self._options[k] = v

    # change how much data we should output
    self._options['verbosity'] += self._options['verbosity_modifier']

    self.CheckConfig()

  def CheckConfig(self):
    """Check the types of configuration parameters, and check for the existance
    of crucial directories."""
    for k, v in self.data_types.iteritems():
      VerifyType(self.Get(k), v)

    for k in ['base_dir', 'test_dir']:
      self.CheckExistance(self._options[k], k)

  def SetAbsolutePath(self, key):
    """For some key, convert the value to an absolute path.

    Args:
      key: the key to convert
    """

    if key not in self._options:
      raise WineTestConfigError('key %s not found' % key)
    self._options[key] = os.path.expanduser(self._options[key])
    if not os.path.isabs(self._options[key]):
      self._options[key] = os.path.join(self._options['base_dir'],
          self._options[key])
    if sys.platform == 'win32':
      self._options[key] = self._options[key].replace('/', '\\')

  def CheckExistance(self, file, keyname=None):
    """Check for the existance of a file.

    Args:
      file: the file to check for
      keyname: optionally, the name of the key that refers to the file
    """

    if not os.path.exists(file):
      if keyname:
        logging.error('%s (%s) does not exist!' % (file, keyname))
      else:
        logging.error('%s does not exist!' % file)
      raise WineTestConfigError('%s does not exist!' % file)


class GetToolsConfig(GenericConfig):

  """The configuration class for the YAWT's get-tools command."""

  def __init__(self, base_dir, command_line_options):
    """Initialize the configuration parameters and check them.

    Args:
      base_dir: the directory containing YAWT
      command_line_options: list of options from the command line
    """
    GenericConfig.__init__(self, base_dir, command_line_options)
    self.data_types = {
      'dry_run': IntType,
      'overwrite': IntType,
    }

    # The tools key is a list of prefixes for programs in tools/ that
    # get-tools care about. For each prefix, these keys should exist:
    # prefix_bin, prefix_url, prefix_sum
    self._options['tools'] = ['autohotkey', 'cabextract', 'regread', 'unzip']
    for k in self._options['tools']:
      self.SetAbsolutePath(k + '_bin')

    self.CheckGetToolsConfig()

  def CheckGetToolsConfig(self):
    """Perform checks on configuration parameters used by the get-tools
    command."""
    for k, v in self.data_types.iteritems():
      VerifyType(self.Get(k), IntType)
    for k in self._options['tools']:
      VerifyType(self.Get(k + '_bin'), StringType)
      VerifyType(self.Get(k + '_url'), StringType)
      if not IsSha1sum(self.Get(k + '_sum')):
        raise WineTestConfigError('%s is an invalid SHA1 checksum' % k + '_sum')


class FetchConfig(GenericConfig):

  """The configuration class for the YAWT's fetch command."""

  def __init__(self, base_dir, command_line_options):
    """Initialize the configuration parameters and check them.

    Args:
      base_dir: the directory containing YAWT
      command_line_options: list of options from the command line
    """
    GenericConfig.__init__(self, base_dir, command_line_options)
    self.data_types = {
      'dry_run': IntType,
      'overwrite': IntType,
    }

    self.CheckFetchConfig()

  def CheckFetchConfig(self):
    """Perform checks on configuration parameters used by the fetch command."""
    for k, v in self.data_types.iteritems():
      VerifyType(self.Get(k), v)


class InstallConfig(GenericConfig):

  """The configuration class for the YAWT's install command.

  This class inherits from GenericConfig and adds specific checks for
  parameters used only by the install command, as well as additional methods
  that are used during the install and depends heavily on the install
  configuration data.
  """

  def __init__(self, base_dir, command_line_options):
    """Initialize the configuration parameters and check them.

    In addition to the GenericConfig __init__ function, this function sets up
    many OS dependent parameters.

    Args:
      base_dir: the directory containing YAWT
      command_line_options: list of options from the command line
    """
    GenericConfig.__init__(self, base_dir, command_line_options)
    self.data_types = {
      'dry_run': IntType,
      'no_verify': IntType,
    }

    self.__saved_environments = {}

    self.__linux_vars = ['wine_bin', 'wineprefixcreate_bin']
    self.__install_vars = ['c_drive', 'autohotkey_bin', 'cabextract_bin',
    'regread_bin', 'unzip_bin']

    if sys.platform == 'linux2':
      # If the configuration does not specify wine_bin/wineprefixcreate_bin,
      # look for them in the PATH
      wine_commands = {'wine_bin': 'wine',
        'wineprefixcreate_bin': 'wineprefixcreate'}
      for var, exe in wine_commands.iteritems():
        if var not in self._options:
          exe_found = False
          if 'PATH' in os.environ:
            for d in os.environ['PATH'].split(':'):
              path = os.path.join(d, exe)
              if os.access(path, os.R_OK|os.X_OK):
                self._options[var] = path
                exe_found = True
                break
          if not exe_found:
            raise WineTestConfigError('%s not in $PATH or config file' % var)

      for k in self.__linux_vars:
        self.SetAbsolutePath(k)
        self.CheckExistance(self._options[k], k)

      # If the configuration does not specify wine_dir, then look for
      # WINEPREFIX. If WINEPREFIX does not exist either, use a random wine_dir.
      self._options['random_wine_dir'] = False
      if 'wine_dir' not in self._options:
        if 'WINEPREFIX' in os.environ:
          self._options['wine_dir'] = os.environ['WINEPREFIX']
        else:
          self._options['random_wine_dir'] = True
          self.CreateRandomWineDir()
      self.SetAbsolutePath('wine_dir')
      self._options['c_drive'] = os.path.join(self._options['wine_dir'],
          'dosdevices/c:')
    elif sys.platform == 'win32':
      # Set the default c_drive on win32
      self._options['c_drive'] = '\\'

    # check the CD path if it is set
    if 'cd_dir' in self._options:
      self.SetAbsolutePath('cd_dir')
      self.CheckExistance(self._options['cd_dir'], 'cd_dir')

    self.__GetWineEnvironment()

    for k in self.__install_vars:
      self.SetAbsolutePath(k)

    self.CheckInstallConfig()
    self.DeleteRandomWineDir()

  def CheckInstallConfig(self):
    """Perform checks on configuration parameters used by the install
    command."""
    for k, v in self.data_types.iteritems():
      VerifyType(self.Get(k), IntType)

    if sys.platform == 'linux2':
      for k in self.__linux_vars:
        self.CheckExistance(self._options[k], k)

    for k in self.__install_vars:
      self.CheckExistance(self._options[k], k)

  def __CreateNewEnvironment(self, wine_vars, env_vars):
    """Create a new environment, adds new variables in the process.

    Args:
      wine_vars: dictionary of Wine specific variables only added on platforms
        using Wine
      env_vars: dictionary of variables added on all platforms

    Returns:
      a dictionary with the new environment.
    """

    environ = copy.deepcopy(os.environ)
    if env_vars:
      for k, v in env_vars.iteritems():
        environ[k] = v

    if sys.platform == 'linux2':
      # set WINEPREFIX
      environ['WINEPREFIX'] = self.Get('wine_dir')
      # add new wine specific environment variables
      if wine_vars:
        for k, v in wine_vars.iteritems():
          environ[k] = v
    return environ

  def __GetWineEnvironment(self):
    env = {}
    if sys.platform == 'linux2':
      command = 'cmd /cset'
      (r, out_data, out_err) = self.RunWindowsCommand(command)
      if r != 0:
        raise WineTestConfigError('failed to get environment variables')
      for line in out_data.splitlines():
        if line.find('=') > 0:
          (k, v) = line.split('=', 1)
          env[k.lower()] = v
    elif sys.platform == 'win32':
      for k, v in os.environ.iteritems():
        env[k.lower()] = v
    self.Set('wine_env', env)

  def RunWindowsCommand(self, command, wine_vars=None, env_vars=None,
      in_data=None):
    """Installers can call this method to run commands without worrying
    about cross-platform issues.

    Args:
      command: the command to run
      wine_vars: optional dictionary of environment variables only used in Wine
      env_vars: optional dictionary of environment variables
      in_data: optional string for the input to the command

    Returns:
      a tuple with:
        the return value from the command
        the output to stdout
        the output to stderr
    """
    shell = True
    environ = self.__CreateNewEnvironment(wine_vars, env_vars)
    stdin = subprocess.PIPE
    # we use temporary files because the pipe on Windows get clogged easily
    stdout = tempfile.TemporaryFile()
    stderr = tempfile.TemporaryFile()

    if sys.platform == 'linux2':
      command = "'%s' %s" % (self.Get('wine_bin'), command)
    elif sys.platform == 'win32':
      shell = False

    logging.debug('running command: %s' % command)
    p = subprocess.Popen(command, env=environ, shell=shell, stdin=stdin,
        stdout=stdout, stderr=stderr)

    if in_data:
      p.stdin.write(in_data)
    p.stdin.close()
    r = p.wait()
    stdout.seek(0)
    stderr.seek(0)
    return (r, stdout.read(), stderr.read())

  def CreateRandomWineDir(self):
    """Randomly create a Wine directory.
    """
    if 'random_wine_dir' in self._options and self._options['random_wine_dir']:
      temp_dir = tempfile.mkdtemp("-winetest")
      r = subprocess.call([self.Get('wineprefixcreate_bin'), '--prefix',
          temp_dir], stdout=None)
      if r != 0:
        raise WineTestConfigError('Error creating temporary Wine directory')

      self._options['wine_dir'] = temp_dir
      self._options['c_drive'] = os.path.join(self._options['wine_dir'],
          'dosdevices/c:')
      self.SetAbsolutePath('wine_dir')
      self.SetAbsolutePath('c_drive')

      # Check and make sure the drive is not on a remotely mounted fs
      p = subprocess.Popen(['stat', '--filesystem', '--format', '%T',
          '--dereference', self._options['c_drive']], stdout=subprocess.PIPE)
      fs_type = p.stdout.read()
      if fs_type in ['nfs', 'smb', 'coda'] or fs_type.find('Unknown') > 0:
        self.DeleteRandomWineDir()
        raise WineTestConfigError('The wine c_drive is on a %s filesystem' %
            fs_type)

  def DeleteRandomWineDir(self):
    """Delete the Wine directory created by CreateRandomWineDir."""
    if 'random_wine_dir' in self._options and self._options['random_wine_dir']:
      # we're about to blow away an entire directory, check to see it ends
      # with '-winetest', since that's the suffix we gave it in
      # CreateRandomWineDir()
      assert self._options['wine_dir'].endswith('-winetest')

      # give wineserver a few seconds to finish, then delete the directory
      wait_time = 5
      time.sleep(wait_time)
      shutil.rmtree(self._options['wine_dir'])
      del self._options['wine_dir']
      del self._options['c_drive']


class AppData(BaseConfig):

  """This class is the container for the information pertaining to a Windows
  application YAWT installs and test.
  """

  def __init__(self, yar_file):
    """Loads the application data from file.

    Args:
      yar_file: path to the YAR file that contains the application test
    """
    BaseConfig.__init__(self)
    self.data_types = {
      'appname': StringType,
      'installable': StringType,
      'profile_name': StringType,
      'sha1sum': StringType,
      'url': StringType,
      'vendor': StringType,
      'version': StringType,
    }

    # extract profile from yar file
    yar = YAR(yar_file)
    profile_data = yar.Read('profile.txt')

    # parse profile
    appdata = WinRegParser(profile_data)
    for k, v in appdata['profile'].iteritems():
      self._options[k] = v[0]

    install_script_data = yar.Read('run_install.py')
    yar.Close()
    self.Set('install_script', install_script_data)
    self.Set('profile_name', '%s_%s_%s' % (self.Get('vendor'),
      self.Get('appname'), self.Get('version')))

    self.ConvertStringsToInts()
    self.CheckConfig()
    self.__ParseCdUri()

  def CheckConfig(self):
    """Perform checks on application data parameters."""
    for k, v in self.data_types.iteritems():
      VerifyType(self.Get(k), v)
    if not IsSha1sum(self.Get('sha1sum')):
      raise WineTestConfigError('%s has an invalid SHA1 checksum' %
          self.Get('profile_name'))

  def ConvertStringsToInts(self):
    """Convert strings containing integer values to ints."""
    for k, v in self.data_types.iteritems():
      if v == IntType and type(self._options[k]) == StringType:
        self._options[k] = int(self._options[k])

  def __ParseCdUri(self):
    """Parse a cd:// URI"""
    is_cd = False
    if self.Get('url').startswith('cd://'):
      is_cd = True
      cd_uri = self.Get('url')[5:]
      cd_uri = cd_uri.replace('/', '\\')
      self.Set('cd_uri', cd_uri)
    self.Set('is_cd', is_cd)


class InstallData(BaseConfig):

  """This class is the container for the information needed to verify Windows
  application installs.
  """

  def __init__(self, yar_file):
    """Loads the installation data from file.

    Args:
      yar_file: path to the YAR file that contains the application test
    """
    BaseConfig.__init__(self)

    # load application data from file
    (reg_data, file_data, reg_todo, file_todo) = LoadInstallData(yar_file)
    self.__ConvertInstallDataFormat(reg_data, file_data, reg_todo, file_todo)

    self.CheckConfig()


  def __ConvertInstallDataFormat(self, reg_data, file_data, reg_todo,
      file_todo):
    """Converts the installation data to data structures YAWT understands.

    Args:
      reg_data: a WinRegParser instance
      file_data: a dictionary of file names and their sha1sums
      reg_todo: a dictionary of registry entries marked as 'todo'
      file_todo: a dictionary of file entries marked as 'todo'
    """
    reg_entries = {}
    encoding = locale.getdefaultlocale()[1]
    if not encoding:
      encoding = 'ISO8859-1'
    for s, key_value in reg_data.iteritems():
      s = s.decode('utf8').encode(encoding)
      for k, v in key_value.iteritems():
        if not k:
          k = ''
        else:
          k = k.decode('utf8').encode(encoding)
        new_key = (s, k)
        if new_key in reg_todo:
          todo = 1
        else:
          todo = 0

        (data, data_type) = v
        if data_type == 'REG_SZ':
          new_data = data.decode('utf8').encode(encoding)
        elif data_type == 'REG_EXPAND_SZ':
          new_data = \
              data[:-1].decode('utf8').encode(encoding)
        elif data_type == 'REG_DWORD':
          d = '%08x' % v[0]
          new_data = ''
          for i in xrange(6, 0, -2):
            new_data += d[i:i+2]
            new_data += ', '
          new_data += d[:2]
        else:
          new_data = reg_data.ConvertStringToCommaSeparatedHex(v[0])
          new_data = new_data.replace(',', ', ')
        reg_entries[new_key] = (new_data, data_type, todo)
    self.Set('reg_entries', reg_entries)

    file_entries = {}
    for k, v in file_data.iteritems():
      if k in file_todo:
        todo = 1
      else:
        todo = 0
      file_entries[k] = (v, todo)
    self.Set('file_entries', file_entries)

  def CheckConfig(self):
    """Perform checks on application data parameters."""
    VerifyType(self.Get('reg_entries'), DictType)
    VerifyType(self.Get('file_entries'), DictType)

    VerifyInstallData(self.Get('reg_entries'), self.Get('file_entries'))


########################################################################
#
# Error Classes
#
########################################################################

class WineTestError(Exception):
  pass

class WineTestConfigError(WineTestError):
  pass

class WineTestLogParseError(WineTestError):
  pass

class WineTestTypeError(WineTestError):
  pass

class WineTestWriteError(WineTestError):
  pass

class WinRegParserFormatError(Exception):
  pass

class YARError(Exception):
  pass

class YARDecodeError(YARError):
  pass

class YARFormatError(YARError):
  pass

class YARInternalError(YARError):
  pass

class YARVersionError(YARError):
  pass


########################################################################
#
# Installable Classes
#
# Foo -> Bar <--> Foo uses Bar
#
#      --------------------> InstallVerifier
#     /
# Installer ---------------> InstallableChecksummer
#                      /
# InstallableFetcher --
#

########################################################################

class ToolFetcher:

  """Download the tools needed to run YAWT."""

  def __init__(self, config):
    """Initializes the class with the YAWT configuration.

    Args:
      config: configuration options (GetToolsConfig class)
    """

    self.config = config

  def FetchTools(self):
    """Fetch (if needed) and checksum the tools.

    Returns:
      True if all the downloads succeed
      False if any of the downloads fail
    """

    # TODO(thestig): work out better behavior for --dry-run
    if self.config.Get('dry_run'):
      return True

    return_status = True

    # make sure directories exists, if not, create them for the user
    for k in self.config.Get('tools'):
      file_name = self.config.Get(k + '_bin')
      dir_name = os.path.dirname(file_name)
      url = self.config.Get(k + '_url')
      sha1sum = self.config.Get(k + '_sum')
      if not os.path.exists(dir_name):
        os.makedirs(dir_name)

      if self.config.Get('overwrite'):
        download_status = DownloadFile(url, file_name)
      else:
        # is the file already there?
        # maybe we do not need to download it
        if VerifySha1sum(file_name, sha1sum):
          continue

        download_status = DownloadFile(url, file_name)

      # if we failed to download the file
      if not download_status:
        return_status = False
        continue

      # check to make sure the downloaded file is intact
      if not VerifySha1sum(file_name, sha1sum):
        return_status = False

    return return_status

class InstallableChecksummer:

  """Checksum the installation executable for some application."""

  def __init__(self, profile, config):
    """Initializes the class with the profile for an application, and the YAWT
    configuration.

    Args:
      profile: an application profile (AppData class)
      config: configuration options (GenericConfig class)
    """
    self.config = config
    self.profile = profile

    self.download_dir = os.path.join(self.config.Get('download_dir'),
        self.profile.Get('profile_name'))
    if self.profile.Get('is_cd'):
      try:
        cd_dir = self.config.Get('cd_dir')
      except WineTestConfigError:
        logging.error('cd_dir needs to be set')
        raise
      self.installable = os.path.join(cd_dir, self.profile.Get('cd_uri'))
    else:
      self.installable = os.path.join(self.download_dir,
          self.profile.Get('installable'))

  def CheckInstallable(self):
    """Check to make sure the installable exists and has the correct checksum.
    """

    logging.debug("checking %s" % self.installable)
    if os.path.exists(self.installable):
      if VerifySha1sum(self.installable, self.profile.Get('sha1sum')):
        return True
      logging.info("%s is corrupt, bad checksum" % self.installable)
      return False
    else:
      logging.info("%s does not exist" % self.installable)
      return False


class InstallableFetcher:

  """Download installables from the web."""

  def __init__(self, profile, config):
    """Initializes the class with the profile for an application, and the YAWT
    configuration.

    Args:
      profile: an application profile (AppData class)
      config: configuration options (FetchConfig class)
    """
    self.config = config
    self.profile = profile

    self.download_dir = os.path.join(self.config.Get('download_dir'),
        self.profile.Get('profile_name'))
    self.installable = os.path.join(self.download_dir,
        self.profile.Get('installable'))
    self.checksummer = InstallableChecksummer(profile, config)

    logging.info("Loading profile: %s" % self.profile.Get('profile_name'))

  def __FetchInstallable(self):
    """Attempt to fetch the executable, always overwrites.

    Returns:
      True if the download succeeds
    """

    return DownloadFile(self.profile.Get('url'), self.installable)

  def __ManualFetchInstallable(self):
    """Tell the user to download the installable.

    Returns:
      Always return False.
    """
    print "Please download %s %s %s to:" % (self.profile.Get('vendor'),
        self.profile.Get('appname'), self.profile.Get('version'))
    print os.path.abspath(self.installable)
    return False

  def FetchInstallable(self):
    """Fetch (if needed) and checksum the installable.

    Returns:
      True if the download succeeds
    """

    if self.config.Get('dry_run'):
      return True

    # handle cd:// uri
    if self.profile.Get('is_cd'):
      return self.checksummer.CheckInstallable()

    # make sure directory exists, if not, create it for the user
    if not os.path.exists(self.download_dir):
      os.makedirs(self.download_dir)

    # select the appropriate fetch function
    if self.profile.Get('url'):
      fetch_function = self.__FetchInstallable
    else:
      fetch_function = self.__ManualFetchInstallable

    if self.config.Get('overwrite'):
      fetched_installable = fetch_function()
    else:
      # is the file already there?
      # maybe we do not need to download it
      if self.checksummer.CheckInstallable():
        return True

      fetched_installable = fetch_function()

    # if we failed to fetch the installable
    if not fetched_installable:
      return False

    # check to make sure the installable is intact
    return self.checksummer.CheckInstallable()


class InstallVerifier:

  """Verify application installations.

  The InstallVerifier uses the InstallData recorded from some previous
  installation to verify the registry keys and the files listed exists, and
  the registry values and file sha1sums are correct.
  """

  def __init__(self, config, installdata):
    """Initializes the class with the YAWT configuration, and the installation
    data for the application.

    Args:
      config: configuration options (InstallConfig class)
      installdata: application installation data (InstallData class)
    """
    self.config = config
    self.installdata = installdata

  def __RunRegread(self):
    """Run the regread tool to query for registry keys.

    Returns:
      a list of values from the registry
    """
    # path for regread should be in quotes if we use long file names
    regread = '"%s"' % self.config.Get('regread_bin')

    # create string with registry keys to query
    str_buf = StringIO.StringIO()
    for k in self.installdata.Get('reg_entries').iterkeys():
      str_buf.write('%s\n%s\n' % k)
    raw_output = self.config.RunWindowsCommand(regread,
        in_data=str_buf.getvalue())[1]
    return raw_output.split('\n')

  def __CompareRegEntries(self, entry, baseline, data_type):
    """Do a fuzzy compare on two registry entries and determine if they are the
    same

    Args:
      entry: the registry value being checked
      baseline: the registry value to check against
      data_type: the value's registry data type

    Returns:
      True if the two values are the same
    """

    # Expand the variable if it's an expandable string
    if data_type == 'REG_EXPAND_SZ':
      env = self.config.Get('wine_env')

      variables_to_expand = []
      expand_prog = re.compile('\%(\w+)\%(.*)')
      expand_string = baseline
      while True:
        search_result = expand_prog.search(expand_string)
        if not search_result:
          break
        (var, expand_string) = search_result.groups()
        variables_to_expand.append(var)
      for var in variables_to_expand:
        if var.lower() in env:
          string_to_replace = '%%%s%%' % var
          baseline = baseline.replace(string_to_replace, env[var.lower()])

    # TODO(thestig):
    # use win32 GetLongPathName() to convert all names to long names
    # Normally apps use long file names in registry keys, but when they use DOS
    # 8.3 names, the 'Program Files' directory shortens to PROGRA~1 on Windows.
    # In Wine, it shortens to PROG~FBU. This causes comparisions for registry
    # entries from the installation logs to the wine registry to fail when
    # they're otherwise correct, aside from this difference. We do this match
    # and replace in order to prevent the comparisions from failing.
    path_replacements = [
      ('C:\\\\PROG~FBU\\\\COMM~CP1\\\\System\\\\OLE_~ETL',
          'C:\\\\Program Files\\\\Common Files\\\\System\\\\OLEDB~1'),
      ('C:\\\\PROG~FBU\\\\COMM~CP1\\\\INST~JM1\\\\engine\\\\6\\\\INTE~MEX',
          'C:\\\\PROGRA~1\\\\COMMON~1\\\\INSTAL~1\\\\engine\\\\6\\\\INTEL3~1'),
      ('C:\\\\PROG~FBU', 'C:\\\\PROGRA~1'),
    ]

    for pattern, replacement in path_replacements:
      prog = re.compile(pattern, re.IGNORECASE)
      if prog.search(entry):
        entry = prog.sub(replacement, entry)
    return entry.lower() == baseline.lower()

  def __ConvertNonPrintableCharacters(self, c):
    if c in string.printable:
      return c
    return ' '

  def VerifyRegistry(self):
    """Verify the registry keys for an installation.

    Returns:
      True if all the registry keys are correct or marked as todo
    """
    reg_entries = self.__RunRegread()
    # TODO(thestig): catch the assert
    assert len(self.installdata.Get('reg_entries')) + 1 == len(reg_entries)

    i = 0
    reg_correct = 0
    reg_incorrect = 0
    reg_todo = 0
    for (key, name), (value, data_type, todo) in \
        self.installdata.Get('reg_entries').iteritems():
      expected = value.rstrip();
      entry = reg_entries[i].rstrip();
      # InCtrl5 converts some non-printable characters to spaces
      entry = ''.join(self.__ConvertNonPrintableCharacters(c) for c in entry)
      if self.__CompareRegEntries(entry, expected, data_type):
        if todo:
          reg_incorrect += 1
          logging.debug("%s: %s is correct with todo flag" % (key, name))
        else:
          reg_correct += 1
      else:
        if todo:
          reg_todo += 1
          logging.debug("%s: %s is marked as todo" % (key, name))
          logging.debug("%s <--> %s" % (expected, entry))
        else:
          reg_incorrect += 1
          logging.debug("%s: %s is incorrect" % (key, name))
          logging.debug("%s <--> %s" % (expected, entry))
      i += 1
    logging.info("registry entries: %d correct, %d todo, %d incorrect" %
        (reg_correct, reg_todo, reg_incorrect))
    if reg_incorrect == 0:
      return True
    return False

  def VerifyFiles(self):
    """Verify the files for an installation.

    Returns:
      True if all the files are correct or marked as todo
    """
    file_correct = 0
    file_incorrect = 0
    file_todo = 0

    c_colon_slash_prog = re.compile('^c:\\\\')
    slash_prog = re.compile('\\\\')
    for k, v in self.installdata.Get('file_entries').iteritems():
      unix_name = c_colon_slash_prog.sub('', k)
      unix_name = slash_prog.sub('/', unix_name)
      full_path = GetCaseInsensitiveFilename(self.config.Get('c_drive'),
          unix_name)
      (sha1sum, todo) = v
      try:
        result = VerifySha1sum(full_path, sha1sum)
        if result:
          if todo:
            file_incorrect += 1
            logging.debug("%s is correct with todo flag" % (unix_name))
          else:
            file_correct += 1
        else:
          if todo:
            file_todo += 1
            logging.debug("%s has wrong checksum (todo)" % unix_name)
          else:
            file_incorrect += 1
            logging.debug("%s has wrong checksum (incorrect)" % unix_name)
      except IOError, (error, strerror):
        if todo:
          file_todo += 1
        else:
          file_incorrect += 1
          logging.debug("I/O error %s - %s: %s" % (error, full_path, strerror))
        continue

    logging.info("file entries: %d correct, %d todo, %d incorrect" %
        (file_correct, file_todo, file_incorrect))
    if file_incorrect == 0:
      return True
    return False

  def VerifyInstall(self):
    """Verify the installation is correct.

    Returns:
      True if everything is correct/marked as todo
    """


    reg_result = self.VerifyRegistry()
    file_result = self.VerifyFiles()
    return reg_result and file_result


class Installer:

  """Installs Windows Applications."""

  def __init__(self, config, test_name):
    """Initializes the class with the profile for an application, the YAWT
    configuration, and the installation data for the application.

    Args:
      config: configuration options (InstallConfig class)
      test_name: the file path for the test YAR file
    """
    self.config = config
    self.profile = AppData(test_name)
    self.installdata = InstallData(test_name)
    self.checksummer = InstallableChecksummer(self.profile, config)
    self.verifier = InstallVerifier(config, self.installdata)

    self.download_dir = os.path.join(self.config.Get('download_dir'),
        self.profile.Get('profile_name'))
    if self.profile.Get('is_cd'):
      try:
        cd_dir = self.config.Get('cd_dir')
      except WineTestConfigError:
        logging.error('cd_dir needs to be set')
        raise
      self.installable = os.path.join(cd_dir, self.profile.Get('cd_uri'))
    else:
      self.installable = os.path.join(self.download_dir,
          self.profile.Get('installable'))
    yar = YAR(test_name)
    file_list = yar.List()
    ExtractYAR(yar, file_list, self.download_dir, overwrite=True)
    yar.Close()

    logging.info("Loaded profile: %s" % self.profile.Get('profile_name'))

  def __InstallApplication(self):
    """Install the application by loading the install script and running it.

    Returns:
      The return code from the installation script
    """
    script_dict = {}
    exec(self.profile.Get('install_script'), script_dict)

    try:
      run_install_function = script_dict['run_install']
    except KeyError:
      logging.error("%s invalid install script" %
          self.profile.Get('profile_name'))
      return False

    try:
      install_result = run_install_function(self.installable,
          self.profile, self.config)
    except (AssertionError, WineTestConfigError):
      logging.error("%s install script caused an error" %
          self.profile.Get('profile_name'))
      install_result = False

    return install_result

  def InstallApplication(self):
    """Install the application and verify the install afterwards.

    Returns:
      True if the application installs correctly, or the no_verify flag is set.
    """

    if not self.checksummer.CheckInstallable():
      return False

    # Create a new .wine directory if needed
    self.config.CreateRandomWineDir()

    return_value = True
    if not self.config.Get('dry_run'):
      if not self.__InstallApplication():
        logging.info("%s installation failed" %
            self.profile.Get('profile_name'))
        return_value = False

    if return_value:
      if not self.config.Get('no_verify'):
         verify_result = self.verifier.VerifyInstall()
         if verify_result:
           logging.info("%s installation verified" %
               self.profile.Get('profile_name'))
           return_value = True
         else:
           logging.info("%s installation failed verification" %
               self.profile.Get('profile_name'))
           return_value = False

    self.config.DeleteRandomWineDir()
    return return_value


########################################################################
#
# Line Parser Classes
#
########################################################################

class LineSeparatedDataParser:

  """This class reads in lines of text and produces a dictionary.
  
  Each line is a key in the dictionary.
  """

  def __init__(self, data):
    """Initializes the LineSeparatedDataParser class.
    
    Args:
      data: the data to parse
    """
    self._dict = {}
    self.ParseData(data)

  def ParseData(self, data):
    """Parses the data and put the entries in the dictionary.
    
    Args:
      data: the data to parse
    """
    lines = self.RemoveComments(data.splitlines())
    for f in lines:
      self._dict[f] = None

  def RemoveComments(self, lines):
    """Removes lines that starts with '#' or ';'.
    
    Args:
      lines: a list of strings

    Returns:
      a list with only the strings that are not comments
    """
    out = []
    for i in xrange(len(lines)):
      entry = lines[i].strip()
      if not entry.startswith('#') and not entry.startswith(';'):
        out.append(entry)
    return out

  """These functions are pass-throughs to self._dict so class instances
  behaves like a dictionary.
  """

  def __contains__(self, key):
    return key in self._dict

  def __getitem__(self, key):
    return self._dict[key]

  def __len__(self):
    return len(self._dict)

  def get(self, key):
    return self._dict.get(key)

  def has_key(self, key):
    return self._dict.has_key(key)

  def items(self):
    return self._dict.items()

  def iteritems(self):
    return self._dict.iteritems()

  def iterkeys(self):
    return self._dict.iterkeys()

  def itervalues(self):
    return self._dict.itervalues()

  def keys(self):
    return self._dict.keys()

  def values(self):
    return self._dict.values()


class LineSeparatedKeyValueParser(LineSeparatedDataParser):

  """This class reads in lines of text and produces a dictionary.
  
  Every pair of lines represents the key and value for a dictionary entry.
  """

  def ParseData(self, data):
    """Parses the data as line separated key/value pairs.
    
    Args:
      data: the data to parse
    """
    lines = self.RemoveComments(data.splitlines())
    for i in xrange(0, len(lines), 2):
      self._dict[lines[i]] = lines[i + 1]


class RegTodoEntriesParser(LineSeparatedDataParser):

  """This class reads in lines of text and produces a dictionary.
  
  Every pair of lines represents a tuple that goes into the dictionary.
  """

  def ParseData(self, data):
    """Parses the data and put the entries in the dictionary.
    
    Args:
      data: the data to parse
    """
    lines = self.RemoveComments(data.splitlines())
    for i in xrange(0, len(lines), 2):
      section = lines[i]
      key = lines[i + 1]
      self._dict[(section, key)] = None


########################################################################
#
# WinRegParser Class
#
########################################################################

class WinRegParser:

  """Reads Window Registry Files (version 4).

  This class reads Window Registry Files (.reg) and makes them accessible
  as a dictionary of dictionaries. The keys in the top-most dictionary are the
  different registry section names. ("keys" in Window terminology) The values
  corresponding to these keys are dictionaries that contain key-values pairs
  for that section. ("values" and "data" in Window terminology.)
  WinRegParser can also write its content out to a Windows Registry File.

  This implementation supports the following registry data types:
  - REG_SZ (string)
  - REG_MULTI_SZ (multiple string array)
  - REG_BINARY (binary data)
  - REG_DWORD (a 32 bit value)
  - REG_EXPAND_SZ ("expandable" string)
  - REG_NONE (no defined value type)
  """

  def __init__(self, reg_data=None):
    self.valid_chars = string.ascii_letters + string.digits + string.punctuation
    self.sections = {}
    if reg_data:
      self._ParseInput(reg_data)

  def ConvertCommaSeparatedHexToString(self, line):
    """Converts a string of comma separated hex values into a string.

    Args:
      line: a line of comma separated hex values

    Returns:
      a string containing the hex values
    """
    # get rid of whitespaces
    line = re.sub('\s', '', line)
    if not line:
      return ''

    r = StringIO.StringIO()
    # separate values by comma
    for f in line.split(','):
      # convert each value into a character and append it
      try:
        r.write(chr(int(f, 16)))
      except ValueError:
        raise WinRegParserFormatError('Invalid hex value: %s' % f)
    return r.getvalue()

  def ConvertStringToCommaSeparatedHex(self, line):
    """Converts a string into a string of comma separated hex values.

    Args:
      line: a string

    Returns:
      a string of comma separated hex values
    """
    if not line:
      return ''
    r = StringIO.StringIO()
    # separate values by comma
    for i in xrange(len(line) - 1):
      # convert each character into a value and append it
      try:
        r.write("%02x," % ord(line[i]))
      except TypeError:
        raise WinRegParserFormatError('Invalid value: %s' % line[i])
    r.write("%02x" % ord(line[-1]))
    return r.getvalue()

  def _GetSectionName(self, line):
    """Extracts the section name from a registry section.

    Args:
      line: a line that contains the section name

    Returns:
      A string with the section name
    """
    if not self._IsSection(line):
      raise WinRegParserFormatError('Not a section: %s' % line)

    # get rid of brackets
    line_stripped = line[1:-1]
    self._ValidateSectionName(line_stripped)
    return line_stripped

  def _GetKeyValue(self, line):
    """Extracts the registry key/value pair from a line.

    Args:
      line: the line that contains a key/value pair

    Returns:
      a tuple with the key and (value, value type) tuple
    """
    string_pat = '"(?:[^\\\\"]|\\\\\\\\|\\\\")'
    key_string_pat = string_pat + '+"'
    value_string_pat = string_pat + '*"'
    hex_value_pat = '(?:\s*[0-9a-fA-F]{2}\s*,\s*)*\s*[0-9a-fA-F]{2}'
    hex_pat = 'hex:(?:%s)?' % hex_value_pat
    multi_pat = 'hex\\(7\\):%s' % hex_value_pat
    expand_pat = 'hex\\(2\\):%s' % hex_value_pat
    none_pat = 'hex\\(0\\):(?:%s)?' % hex_value_pat
    dword_pat = 'dword:\s*[0-9a-fA-F]{1,8}'
    key_pat = '@|%s' % key_string_pat
    value_pat = '(%s)|(%s)|(%s)|(%s)|(%s)|(%s)' % (value_string_pat,
        hex_pat, multi_pat, expand_pat, dword_pat, none_pat)

    key_value_match = re.match('(%s)=(%s)$' % (key_pat, value_pat), line)
    if not key_value_match:
      raise WinRegParserFormatError('Badly formatted key/value pair: %s' % line)
    (k, v, str_v, hex_v, multi_v, expand_v, dword_v, none_v) = \
        key_value_match.groups()

    # map k to dictionary key
    if k == '@':
      k = None
    else:
      # strip quotes, unescape
      k = k[1:-1].decode('string_escape')
    self._ValidateKeyName(k)

    # parse value, assign value_type
    if str_v:
      value_type = 'REG_SZ'
      v = str_v[1:-1].decode('string_escape')
    elif hex_v:
      value_type = 'REG_BINARY'
      v = self.ConvertCommaSeparatedHexToString(hex_v[4:])
    elif multi_v:
      value_type = 'REG_MULTI_SZ'
      v = self.ConvertCommaSeparatedHexToString(multi_v[7:])
    elif expand_v:
      value_type = 'REG_EXPAND_SZ'
      v = self.ConvertCommaSeparatedHexToString(expand_v[7:])
    elif none_v:
      value_type = 'REG_NONE'
      v = self.ConvertCommaSeparatedHexToString(none_v[7:])
    elif dword_v:
      value_type = 'REG_DWORD'
      try:
        v = int(dword_v[6:], 16)
      except ValueError:
        raise WinRegParserFormatError('Invalid DWORD value: %s' % dword_v[6:])
    return (k, (v, value_type))

  def _IsBlank(self, line):
    """Returns True if the line is does not have printable characters."""
    if type(line) != StringType:
      raise TypeError
    for c in line:
      if c in self.valid_chars:
        return False
    return True

  def _IsComment(self, line):
    """Returns True if the line is a comment."""
    if type(line) != StringType:
      raise TypeError
    return line[0] == ';' or line[0] == '#'

  def _IsLineJoin(self, line):
    """Returns True if the line is a line join (ends with \)."""
    if type(line) != StringType:
      raise TypeError
    return line[-1] == '\\'

  def _IsSection(self, line):
    """Returns True if the line is a registry section."""
    if type(line) != StringType:
      raise TypeError
    return len(line) > 2 and line[0] == '[' and line[-1] == ']'

  def _ParseInput(self, reg_data):
    """Validate the input, parse it and store it internally

    Args:
      reg_data: the content of a registry file
    """
    seen_section = False
    line_join = ''

    # split the lines and go through them
    lines = reg_data.splitlines()
    for cur_line in lines:
      # get rid of extra leading / trailing spaces
      cur_line = cur_line.strip()
      # skip blank lines and comments
      if self._IsBlank(cur_line) or self._IsComment(cur_line):
        if line_join:
          raise WinRegParserFormatError('Bad formatting after line join')
        continue
      # set section_name when we find a section and initialize its dictionary
      if self._IsSection(cur_line):
        if line_join:
          raise WinRegParserFormatError('Bad formatting after line join')
        section_name = self._GetSectionName(cur_line)
        if section_name in self.sections:
          raise WinRegParserFormatError('Duplicate Secton: %s' % section_name)
        seen_section = True
        self.sections[section_name] = {}
        continue

      # assume we're dealing with a key/value pair
      if not seen_section:
        raise WinRegParserFormatError('No Section associated with Key')

      # if the line continue onto the next one
      if self._IsLineJoin(cur_line):
        # append the line to line_join without the \ at the end
        line_join += cur_line[:-1] + ' '
        continue

      # finally, we have just one (big long) line
      cur_line = line_join + cur_line
      line_join = ''
      (k, v) = self._GetKeyValue(cur_line)
      if k in self.sections[section_name]:
        raise WinRegParserFormatError('Duplicate Key: %s' % k)
      self.sections[section_name][k] = v

  def _ValidateKeyName(self, name):
    """Check to see if a key name is valid.

    Args:
      name: the key name string
    """
    if name == None:
      return
    if type(name) != StringType:
      raise WinRegParserFormatError('Invalid Section - not a string')
    if not name:
      raise WinRegParserFormatError('Invalid Section - empty name')

  def _ValidateSectionName(self, name):
    """Check to see if a section name is valid.

    Args:
      name: the section name string
    """
    if type(name) != StringType:
      raise WinRegParserFormatError('Invalid Section - not a string')
    if not name:
      raise WinRegParserFormatError('Invalid Section - empty name')
    if name[0] == '\\' or name[-1] == '\\':
      raise WinRegParserFormatError('Invalid Section - starts/ends with \\')
    if '\\\\' in name:
      raise WinRegParserFormatError('Invalid Section - two consecutive \\')

  def AddEntry(self, section, key, value, data_type):
    """Add a new registry entry.

    Args:
      section: the section the key goes into
      key: the name of the key (string, can be None)
      value: REG_SZ/REG_BINARY/REG_EXPAND_SZ/REG_NONE - arbitrary string
             REG_MULTI_SZ - list of strings
             REG_DWORD - integer value
      data_type: 'REG_SZ', 'REG_BINARY', 'REG_MULTI_SZ', 'REG_EXPAND_SZ',
                 'REG_DWORD', or 'REG_NONE'
    """
    self._ValidateSectionName(section)
    self._ValidateKeyName(key)

    if data_type in ['REG_SZ', 'REG_BINARY', 'REG_NONE']:
      if type(value) != StringType:
        raise WinRegParserFormatError('Bad data type')
      v = (value, data_type)
    elif data_type == 'REG_EXPAND_SZ':
      if type(value) != StringType:
        raise WinRegParserFormatError('Bad data type')
      v = (value + '\x00', data_type)
    elif data_type == 'REG_MULTI_SZ':
      if type(value) != ListType:
        raise WinRegParserFormatError('Bad data type')
      buf = StringIO.StringIO()
      for f in value:
        VerifyType(f, StringType)
        buf.write(f)
        buf.write('\x00')
      buf.write('\x00')
      v = (buf.getvalue(), data_type)
    elif data_type == 'REG_DWORD':
      if type(value) != IntType and type(value) != LongType:
        raise WinRegParserFormatError('Bad data type')
      if value < 0 or value > 0xffffffff:
        raise WinRegParserFormatError('Invalid DWORD value')
      v = (value, data_type)
    else:
      raise WinRegParserFormatError('Unknown/Unsupported data type')

    if section in self.sections:
      self.sections[section][key] = v
    else:
      self.sections[section] = {key: v}

  def WriteOutput(self):
    """Returns the internal data as a registry file."""
    output = StringIO.StringIO()
    sections = self.sections.keys()
    sections.sort()
    for s in sections:
      output.write('[%s]\n' % s)
      key_value = self.sections[s]
      keys = key_value.keys()
      keys.sort()
      for k in keys:
        (v, t) = key_value[k]
        if not k:
          output.write('@=')
        else:
          k = k.replace('\\', '\\\\')
          k = k.replace('"', '\\"')
          output.write('"%s"=' % k)

        if t == 'REG_SZ':
          v = v.replace('\\', '\\\\')
          v = v.replace('"', '\\"')
          output.write('"%s"' % v)
        elif t == 'REG_BINARY':
          output.write("hex:%s" % self.ConvertStringToCommaSeparatedHex(v))
        elif t == 'REG_MULTI_SZ':
          output.write("hex(7):%s" % self.ConvertStringToCommaSeparatedHex(v))
        elif t == 'REG_EXPAND_SZ':
          output.write("hex(2):%s" % self.ConvertStringToCommaSeparatedHex(v))
        elif t == 'REG_NONE':
          output.write("hex(0):%s" % self.ConvertStringToCommaSeparatedHex(v))
        elif t == 'REG_DWORD':
          output.write('dword:%08x' % v)
        output.write('\n')
      output.write('\n')
    return output.getvalue()


  """These functions are pass-throughs to self.sections so class instances
  behaves like a dictionary.
  """

  def __contains__(self, key):
    return key in self.sections

  def __delitem__(self, key):
    del self.sections[key]

  def __getitem__(self, key):
    return self.sections[key]

  def __len__(self):
    return len(self.sections)

  def clear(self):
    return self.sections.clear()

  def get(self, key):
    return self.sections.get(key)

  def has_key(self, key):
    return self.sections.has_key(key)

  def items(self):
    return self.sections.items()

  def iteritems(self):
    return self.sections.iteritems()

  def iterkeys(self):
    return self.sections.iterkeys()

  def itervalues(self):
    return self.sections.itervalues()

  def keys(self):
    return self.sections.keys()

  def values(self):
    return self.sections.values()


########################################################################
#
# YAR Class
#
########################################################################

class YARInfo(BaseConfig):

  """YARInfo contains the attributes for a file contained in a YAR file.

  Use BaseConfig's Get() or GetAll() to read attributes.

  Attributes:
    name: file name
    type: file type
    encoding: file encoding
    eof: eof marker
    offset: byte offset of the beginning of the <file> tag within the YAR file
    length: length of the encoded file data in bytes
    header_len: length of the <file ...> header
  """

  def __init__(self):
    BaseConfig.__init__(self)

  def SetAttributes(self, name, type, encoding, eof, offset, length, header):
    """Set the attributes for the file and make them read-only."""

    self.Set('name', name)
    self.Set('type', type)
    self.Set('encoding', encoding)
    self.Set('eof', eof)
    self.Set('offset', offset)
    self.Set('length', length)
    self.Set('header_len', header)
    self.Lock()


class YAR:

  """This class implements the Yet another ARchive, a human readable format.

  TODO(thestig) move this to the project webpage when it gets created

  This implementation only supports version 1 of the YAR format.

  YAR is a human readable, text-only archive format encoded in UTF-8.
  The format is as follows:
  ---
  <YAR $VERSION>
  <file name="foo" encoding="text" eof="XYZ">
  ...
  </file XYZ>
  <file name="bar">
  ...
  </file>
  ...
  <file name="xyzzy" type="directory">
  ...
  </file>
  ---

  Overall Format:
    - The first line of a YAR file is a header with a version number
     - The header cannot contain extra whitespaces
     - The version number is a positive integer greater than 0
    - Following the first line is any number of files, each of which is
      enclosed in between a line that starts with <file ...> and a line that
      starts with </file ...>
    - Any data besides the header that is not enclosed within <file> and
      </file> is ignored.

  File Tags:
    - Every <file> or </file> tag should be exactly one line long
    - For any <file> tag, there can be any number of attributes contained
      within the brackets in the form of attribute = value
      - Attributes and values with whitespace can be contained within single or
        double quotes
      - Attributes and values do not support backslash-escaping. (i.e. \n)
      - Valid attributes are:
        - name: (Required) the filename, with the forward-slash (/) as the
                directory separator.
          - filenames should not contain the '\' character
          - filenames should not contain ':' or start with some form of:
            "../foo", "./foo/../../foo", etc...
          - invalid filenames will cause undefined behavior on systems that
            do not support them
          - all file paths are relative, absolute path will be treated as
            though they do have the any leading forward-slashes
          - trailing forward-slashes in file names are ignored
        - type: (Optional) the file type, either "file" or "directory"
          - if not specified, assume file
          - the content of any directory file is ignored
        - encoding: (Optional) the file encoding, currently we support
                    utf8, base64, and hex
          - if not specified, assume ascii
        - eof: Required only to uniquely identify a file that contains lines
               that look like </file...> i.e. if eof="foo", then the matching
               </file> tag should be </file foo>
          - if not specified, assume </file> is the end tag
      - Unrecognized attributes are ignored
  """

  def __init__(self, yar_file, mode='r', binary_encoding='base64'):
    """Initializes a YAR object

    Args:
      yar_file: a path to a file name (string) or a file-like object
      mode: if yar_file is a file name, 'r' to read, 'w' to overwrite,
            and 'a' to append
      binary_encoding: default encoding to use for binary files
    """
    self.version = 1
    self.valid_modes = {'r': re.compile('[rwa](\+?b|b\+?)'),
      'w': re.compile('w(\+b|b\+)'),
      'a': re.compile('a(\+b|b\+)')}
    self.default_open_modes = {'r': 'rb', 'w': 'wb+', 'a': 'ab+'}
    self.valid_types = ['file', 'directory']
    self.valid_binary_encodings = ['base64', 'hex']
    self.valid_text_encodings = ['utf8']
    self.valid_encodings = (self.valid_binary_encodings
        + self.valid_text_encodings)
    self.valid_attributes = ['name', 'type', 'encoding', 'eof']
    self.random = random.Random()
    self.closed = False
    self.mode = mode

    # validate arguments
    if self.mode not in self.valid_modes.keys():
      raise IOError('invalid mode')
    if binary_encoding not in self.valid_binary_encodings:
      raise YARFormatError('invalid binary encoding')
    self.binary_encoding = binary_encoding

    # determine whether yar_file is a name or file-like object
    if isinstance(yar_file, basestring):
      self.yar_file = open(yar_file, self.default_open_modes[self.mode])
    elif IsFileLikeObject(yar_file):
      # check mode for file-like objects
      try:
        if not self.valid_modes[self.mode].match(yar_file.mode):
          raise IOError('wrong file mode')
      except AttributeError:
        # some file-like objects don't have mode set, so we skip this check
        pass
      self.yar_file = yar_file
    else:
      raise TypeError('yar_file is not a string or file-like object')

    # write a YAR header if overwriting
    if self.mode == 'w':
      self._WriteHeader()
    self._List()

  def Close(self):
    """Close the file, disallow farther read/writes, release the file handle"""
    # allow closing repeatedly, just like file objects
    if self.closed:
      return
    self.yar_file.close()
    self.closed = True

  def Read(self, filename):
    """Reads a file from the archive.

    Args:
      filename: name of the file to read

    Returns:
      File data if file exists, else None
    """
    if self.closed:
      raise IOError("Yar object is closed")

    if filename in self.file_dict:
      return self._ReadFileContent(self.file_dict[filename])
    return None

  def Write(self, out_file, filename, binary_encoding=None, is_dir=False):
    """Append a file to the archive

    Args:
      out_file: a file-like object to write to the archive
      filename: the name of the file inside the archive
      binary_encoding: the binary encoding to use
      is_dir: to append a directory, set to True and leave out_file = None
    """
    if self.closed:
      raise IOError('Yar object is already closed')

    # check parameters
    if self.mode == 'r':
      raise IOError('Archive is read-only')
    if not is_dir and not IsFileLikeObject(out_file):
      raise TypeError('outfile is not a file-like object')
    filename = self._CheckAndCleanupName(filename)
    if filename in self.file_list:
      raise YARFormatError('YAR file cannot contain duplicate names: %s' %
          filename)
    if binary_encoding:
      if binary_encoding not in self.valid_binary_encodings:
        raise YARFormatError('invalid binary encoding')
    else:
      binary_encoding = self.binary_encoding

    # make sure we're at the end of the file
    self.yar_file.seek(0, 2)
    start_position = self.yar_file.tell()
    # if it's a directory, write tag and be done with it
    if is_dir:
      start_tag = self._GenerateFileStartTag(filename, 'directory')
      stop_tag = self._GenerateFileStopTag()
      self.yar_file.write(start_tag)
      self.yar_file.write(stop_tag)

      # update the index
      self.file_dict[filename] = YARInfo()
      self.file_dict[filename].SetAttributes(filename, 'directory', 'utf8',
          None, start_position, 0, len(start_tag))
      self.file_list.append(filename)
      return

    # determine whether data is binary or text, find EOF for text files
    bad_eofs = []
    is_binary = False
    data = ''
    eof_prog = re.compile('^</file(.*)>\s*$')

    while True:
      line = out_file.readline()
      if not line:
        break
      data += line
      is_binary = IsBinary(line)
      if is_binary:
        break
      else:
        # might be a text file, look for strings like </file ...> that would
        # require a EOF to disambiguate it
        eof_match = eof_prog.match(line)
        if eof_match:
          bad_eofs.append(eof_match.groups()[0].rstrip())

    encoding = 'utf8'
    eof = None
    if is_binary:
      # binary data, read in the rest of the file and encode it
      encoding = binary_encoding
      encoder = codecs.getencoder(encoding)
      data += out_file.read()
      data = encoder(data)[0]
      # we may need to add an extra carriage return
      if data[-1] != '\n':
        data += '\n'
      start_tag = self._GenerateFileStartTag(filename, encoding=encoding)
      stop_tag = self._GenerateFileStopTag()
    else:
      # generate an EOF if needed
      eof = ''
      letters_len = len(string.ascii_letters)
      while eof in bad_eofs:
        eof += string.ascii_letters[letters_len * int(random.random())]
      start_tag = self._GenerateFileStartTag(filename, eof=eof)
      stop_tag = self._GenerateFileStopTag(eof)

    # append to out_data
    self.yar_file.write(start_tag)
    self.yar_file.write(data)
    self.yar_file.write(stop_tag)

    # update the index
    self.file_dict[filename] = YARInfo()
    self.file_dict[filename].SetAttributes(filename, 'file', encoding, eof,
        start_position, len(data), len(start_tag))
    self.file_list.append(filename)

  def _GenerateFileStartTag(self, name, filetype=None, encoding=None, eof=None):
    """Generate a <file> tag"""
    if not name:
      raise YARFormatError("No name specified")
    if filetype and filetype not in self.valid_types:
      raise YARFormatError("Invalid file type: %s" % filetype)
    if encoding and encoding not in self.valid_encodings:
      raise YARFormatError("Invalid encoding: %s" % encoding)

    tag = '<file name="%s"' % name
    if filetype:
      tag += ' type="%s"' % filetype
    if encoding:
      tag += ' encoding="%s"' % encoding
    if eof:
      tag += ' eof="%s"' % eof
    tag += '>\n'
    return tag

  def _GenerateFileStopTag(self, eof=None):
    """Generate a </file> tag"""
    if eof:
      return '</file %s>\n' % eof
    return '</file>\n'

  def List(self):
    """Return a list of tuples for the files inside the archive. Each tuple
    contains the file name and file type. The list should be ordered the same
    way the files are listed in the archive."""
    if self.closed:
      raise IOError("Yar object is closed")

    files = []
    for f in self.file_list:
      t = (f, self.file_dict[f].Get('type'))
      files.append(t)

    return files

  def _List(self):
    """Generate YARInfo objects for all the files inside a YAR file.

    YAR files don't have metadata to indicate the length and offsets of the
    files they contain. Therefore, to find a given file inside the archive
    would require scanning through the entire file until its found. This
    function reads through YAR files once and generate YARInfo objects to
    cache the metadata so that we can randomly access files within the
    archive later without having to read the entire file sequencially.
    """

    file_version = self._ReadValidateHeader()
    if not file_version:
      raise YARFormatError("Not a valid YAR file")
    elif file_version != self.version:
      raise YARVersionError("Sorry, we do not support YAR version %d" %
          file_version)

    # this is the list of files in the original order
    self.file_list = []
    # this dictionary contains file name to YARInfo mappings
    self.file_dict = {}
    while True:
      info = self._ReadFileInfo()
      if not info:
        break
      filename = info.Get('name')
      if filename in self.file_dict:
        raise YARFormatError("YAR file contain duplicate names: %s" % filename)
      self.file_list.append(filename)
      self.file_dict[info.Get('name')] = info

  def _WriteHeader(self):
    """Seek to the beginning of file, truncate, and write the YAR header."""
    self.yar_file.seek(0)
    self.yar_file.truncate()
    self.yar_file.write('<YAR %d>\n' % self.version)

  def _ReadValidateHeader(self):
    """Read the first line of the YAR file and determine whether it's valid.

    Returns:
      The version number, or 0 if no valid version number
    """
    self.yar_file.seek(0)
    header = self.yar_file.readline()
    header_match = re.match("<YAR (\d+)>\s$", header)
    if header_match:
      return int(header_match.groups()[0])
    return 0

  def _ReadFileInfo(self):
    """Read the metadata for a single file contained inside a YAR file

    Returns:
      a YARInfo object
    """
    end_line = ''
    start_position = -1
    file_header_length = 0
    attributes = {}

    while True:
      current_position = self.yar_file.tell()
      raw_line = self.yar_file.readline()
      current_line = raw_line.replace('\r\n', '\n')
      if current_line == '':
        return None

      if start_position < 0:
        # look for the <file ...> line
        if current_line.startswith('<file ') and current_line.endswith('>\n'):
          # found it
          start_position = current_position
          file_header_length = len(raw_line)
          # current_line[6:-2] removes '<file ' and '>\n' from <file ...>\n
          attributes = self._ParseAttributes(current_line[6:-2])

          # check parsing result
          if 'name' not in attributes:
            raise YARFormatError("No filename specified")
          attributes['name'] = self._CheckAndCleanupName(attributes['name'])
          if not attributes['name']:
            raise YARFormatError("No filename specified")

          if attributes['encoding'] not in self.valid_encodings:
            raise YARFormatError("file encoding %s not supported" %
                attributes['encoding'])

          if attributes['type'] not in self.valid_types:
            raise YARFormatError('file type %s not supported' %
                attributes['type'])

          # set the </file ...> marker to look for
          end_line = self._GenerateFileStopTag(attributes['eof'])
      else:
        # looking for </file ...> line
        if current_line == end_line:
          # found the end marker
          info = YARInfo()
          length = current_position - start_position - file_header_length
          info.SetAttributes(attributes['name'], attributes['type'],
              attributes['encoding'], attributes['eof'], start_position,
              length, file_header_length)
          return info

  def _ReadFileContent(self, yar_info):
    """Read the content for a file based on the information from yar_info

    Args:
      yar_info: a YARInfo object describing the file to read

    Returns:
      the file data, or an empty string for directories
    """
    # validate input
    file_type = yar_info.Get('type')
    encoding = yar_info.Get('encoding')
    if file_type not in self.valid_types:
      raise YARFormatError("Invalid file type: %s" % file_type)
    if encoding not in self.valid_encodings:
      raise YARFormatError("Invalid encoding: %s" % encoding)

    if file_type == 'directory':
      return ''

    # seek to the beginning of the data
    self.yar_file.seek(yar_info.Get('offset') + yar_info.Get('header_len'))
    raw_data = self.yar_file.read(yar_info.Get('length'))

    raw_data = raw_data.replace('\r\n', '\n')
    if encoding in self.valid_text_encodings:
      return raw_data

    if encoding == 'base64':
      try:
        decoder = codecs.getdecoder('base64')
        return decoder(raw_data)[0]
      except LookupError:
        raise YARInternalError("Internal error, no base64 decoder")
      except binascii.Error, error_msg:
        raise YARDecodeError("Error decoding base64: %s" % error_msg)
    elif encoding == 'hex':
      try:
        decoder = codecs.getdecoder('hex')
        return decoder(raw_data.replace('\n', ''))[0]
      except LookupError:
        raise YARInternalError("Internal error, no hex decoder")
      except TypeError, error_msg:
        raise YARDecodeError("Error decoding hex: %s" % error_msg)
    else:
      # this really shouldn't happen unless self.valid_encodings or
      # self.valid_binary_encodings is wrong somehow, or we forgot to check
      # for it here.
      raise YARInternalError("Internal error, unrecognized encoding")

  def _ParseAttributes(self, line):
    """Parse the attributes from <file > tag in a Yet another ARchive

    Args:
      line: the '...' section of a <file ...> tag

    Returns:
      a dictionary of valid key/value pairs, which always include the 'name'
      key, or an empty dictionary on error
    """
    attributes = {}
    original_line = line

    # search for: a string without ' and ",
    # or a string without ' inside ''
    # or a string without " inside ""
    string_pat = "([^'\"\s]+|'[^']+'|\"[^\"]+\")"

    # = with any number of whitespaces on its sides
    equal_pat = "\s*=\s*"

    # either the end of the string or a whitespace followed by anything
    end_pat = "($|\s.*)"

    quotes_regex = re.compile(string_pat + equal_pat + string_pat + end_pat)
    key = ''
    value = ''

    # start parsing
    while True:
      line = line.lstrip()
      if len(line) <= 0:
        break

      match_result = quotes_regex.match(line)
      if match_result:
        (key, value, line) = match_result.groups()
      else:
        raise YARFormatError('Cannot parse tag: <file %s>' % original_line)

      # get key/value and remove quotes / trailing spaces
      key = self._StripQuotes(key)
      key = key.lower()
      value = self._StripQuotes(value)

      # add key/value to attributes dictionary
      if key in attributes:
        logging.warning('Warning, key %s already exists, overwriting' % key)
      attributes[key] = value

    # done parsing, look for attributes we care about
    if 'name' not in attributes:
      return {}

    # set default return values
    return_values = {'encoding': 'utf8', 'eof': '', 'type': 'file'}
    # go through the keys we care about, if they exist in attributes,
    # then copy key/value to return_values
    for f in self.valid_attributes:
      if f in attributes:
        return_values[f] = attributes[f]
    return return_values

  def _CheckAndCleanupName(self, name):
    """Take a filename, check for invalid characters, normalize it and
    strip of leading/trailing '/'s"""

    if '\\' in name or ':' in name:
      raise YARFormatError("Bad Filename: %s", name)
    new_name = os.path.normpath(name)
    new_name = new_name.replace('\\', '/')
    if new_name.startswith('../'):
      raise YARFormatError("Bad Filename: %s", name)
    return new_name.strip('/')

  def _StripQuotes(self, line):
    """Strip matching (double)quotes from the beginning/end of the line, and
    then strip any additional whitespace from the beginning/end of the line.

    Args:
      line: A line of text, thay may start and end with (double) quotes

    Returns:
      A string with out the leading/trailing (double) quote and whitespace
    """
    if type(line) != StringType:
        raise YARInternalError('Internal Error In _StripQuotes()')
    if line[0] == "'" or line[0] == '"':
      assert line[-1] == line[0]
      line = line[1:-1]
    return line.strip()


########################################################################
#
# Log Parsing Functions
#
########################################################################

def FindCommonDictionary(left, right):
  """Find the key/value pairs two dictionaries have in common.

  Args:
    left: a dictionary
    right: a dictionary

  Returns:
    A list of keys that are in both dictionaries and have the same value.
  """
  same = []
  for k, v in left.iteritems():
    if k in right and right[k] == v:
      same.append(k)
  return same


def ParseInctrlLogRegEntries(lines):
  """Extract all the installation data registry values from a InCtrl5 log.

  Args:
    lines: a list consisting of the lines from the logfile

  Returns:
    A WinRegParser instance with the registry data
  """
  reg_entries = WinRegParser()

  # find the section header
  header_found = False
  header_prog = re.compile('^Values added: [0-9]+$')
  for i in xrange(len(lines)):
    if header_prog.search(lines[i]):
      header_found = True
      break

  if not header_found:
    raise WineTestLogParseError('InCtrl5 log registry header not found')

  lineno = i + 2

  hkey_prog = re.compile('^\tHKEY_')
  key_prog = re.compile('^\t')
  value_prog = re.compile('"$')
  data_prog = re.compile('^\t\tData: ')
  type_prog = re.compile('^\t\tType: ')
  split_re = re.compile('(^.*) \"(.*\"$)')

  # process entries
  while True:
    m = split_re.match(lines[lineno])
    if not m:
      break
    key_value = m.groups();

    # make sure the data is there
    if (not key_value
        or not hkey_prog.search(key_value[0])
        or not value_prog.search(key_value[1])):
      raise WineTestLogParseError('InCtrl5 log contains bad registry \
          key/value on line %d' % lineno)
    if not type_prog.search(lines[lineno + 1]):
      raise WineTestLogParseError('InCtrl5 log contains bad registry data \
          type on line %d' % (lineno + 1))
    if not data_prog.search(lines[lineno + 2]):
      raise WineTestLogParseError('InCtrl5 log contains bad registry data on \
          line %d' % (lineno + 2))

    # get the data
    key = key_prog.sub('', key_value[0])
    value = value_prog.sub('', key_value[1])
    data_type = type_prog.sub('', lines[lineno + 1])
    data = data_prog.sub('', lines[lineno + 2])

    lineno += 3
    if value == '(Default)':
      value = None

    # If the registry data is too long, InCtrl5 displays "data too large"
    if data.startswith('(data too large: '):
      continue
    elif data_type == 'REG_BINARY':
      converted_data = reg_entries.ConvertCommaSeparatedHexToString(data)
    elif data_type == 'REG_DWORD':
      csv = data.split(', ')
      converted_data = 0
      for i in xrange(3, -1, -1):
        converted_data = converted_data * 0x100 + int(csv[i], 16)
    elif data_type in ['REG_SZ', 'REG_EXPAND_SZ', 'REG_NONE']:
      converted_data = data
    elif data_type == 'REG_MULTI_SZ':
      converted_data = []
      for d in data.split('00, ')[:-1]:
        d = d.rstrip(', ')
        converted_data.append(reg_entries.ConvertCommaSeparatedHexToString(d))
    else:
      continue

    reg_entries.AddEntry(key, value, converted_data, data_type)

  return reg_entries


def ParseInctrlLogFileEntries(lines):
  """Extract all the installation data file entries from a InCtrl5 log.

  Args:
    lines: a list consisting of the lines from the logfile

  Returns:
    A dictionary of filename, sha1sum pairs
  """
  file_entries = {}

  # find the section header
  header_found = False
  header_prog = re.compile('^Files added: [0-9]+$')
  for i in xrange(len(lines)):
    if header_prog.search(lines[i]):
      header_found = True
      break

  # get the number of entries in this section
  if not header_found:
    raise WineTestLogParseError('InCtrl5 log file header not found')
  lineno = i + 2

  file_prog = re.compile('^\t')

  # process entries
  while True:
    # get the data
    name = file_prog.sub('', lines[lineno])
    if not name:
      break
    local_name = name.decode('utf8').encode(locale.getdefaultlocale()[1])

    # sha1sum the file
    try:
      file_handle = open(local_name, 'rb')
      sha1summer = sha.new(file_handle.read())
      file_handle.close()
      digest = sha1summer.hexdigest()
      file_entries[name] = digest
    except IOError, (error, strerror):
      logging.error("I/O error %s - %s: %s" % (error, local_name, strerror))
    lineno = lineno + 3

  return file_entries


def AddAdditionalTestFiles(yar):
  """Add files needed for a YAWT test.

  Args:
    yar: The YAR test file to insert into
  """
  reg_todo = ('# This todo file lists registry entries that are known to be\n'
      '# incorrect when installing under Wine.\n'
      '# Add registry entries below; lines that start with "#" or ";" are '
      'ignored.\n'
      '#\n'
      '# Example:\n'
      '#HKEY_LOCAL_MACHINE\\SOFTWARE\\Foo\n'
      '#value1\n'
      '# blank value indicates Default value\n'
      '#HKEY_LOCAL_MACHINE\\SOFTWARE\\Foo\n'
      '#\n')
  file_todo = ('# This todo file lists file entries that are known to be\n'
      '#incorrect when installing under Wine.\n'
      '# Add filename entries below; lines that start with "#" or ";" are '
      'ignored.\n'
      '#\n'
      '# Example:\n'
      '#c:\\path\\to\\file\n'
      '#c:\\path\\to\\another.txt\n')
  profile = ('# This profile defines parameters for the application test\n'
      '[profile]\n'
      '# appname, vendor, and version are strings that uniquely name an '
      'application.\n'
      '"appname"="Fake App"\n'
      '"vendor"="Fake Company"\n'
      '"version"="1.0"\n\n'
      '# url is the direct download for the application. Leave it blank if\n'
      '# there is no direct download.\n'
      '"url"="http://www.hostname.com/path/to/foo.exe"\n\n'
      '# installable is the name of the installer executable. If url is not\n'
      '# blank, then YAWT can automatically download and save the installer.\n'
      '# If url is blank, then one must manually put get the file.\n'
      '"installable"="foo.exe"\n\n'
      '# sha1sum is the SHA-1 checksum for this file.\n'
      '"sha1sum"="a40d70a6b698416976d6978f9869c4141969bda9"\n')
  run_install = ('#!/usr/bin/python2.4\n\n'
      'def run_install(installable, profile, config):\n'
      '  """Installs a win32 application.\n\n'
      '  Args:\n'
      '    installable: the path to the installer executable\n'
      '    profile: the application profile, it contains the variables set in\n'
      '             profile.txt\n'
      '    config: the YAWT configuration, it contains all the runtime options,'
      '\n'
      '            as well as the RunWindowsCommand() method.\n\n'
      '  Returns: True if install succeeds.\n'
      '  """\n\n'
      '  install_command = \'"%s"\' % installable\n\n'
      '  r = config.RunWindowsCommand(install_command)[0]\n'
      '  if r == 0:\n'
      '    return True\n'
      '  return False\n')

  reg_todo_file = StringIO.StringIO(reg_todo)
  file_todo_file = StringIO.StringIO(file_todo)
  profile_file = StringIO.StringIO(profile)
  run_install_file = StringIO.StringIO(run_install)
  yar.Write(profile_file, 'profile.txt')
  yar.Write(run_install_file, 'run_install.py')
  yar.Write(reg_todo_file, 'yawt_reg_todo.txt')
  yar.Write(file_todo_file, 'yawt_file_todo.txt')


def GenerateYAWTTest(reg_entries, file_entries):
  """Generates a YAR file containing a YAWT test.
  
  Args:
    reg_entries: A WinRegParser instance with the registry data
    file_entries: a dictionary of filename, sha1sum pairs

  Returns:
    The content of the YAWT test
  """
  reg_entries_file = StringIO.StringIO(reg_entries.WriteOutput())
  file_entries_file = StringIO.StringIO()
  for k, v in file_entries.iteritems():
    file_entries_file.write(k)
    file_entries_file.write('\n')
    file_entries_file.write(v)
    file_entries_file.write('\n')
  file_entries_file.seek(0)

  yar_file = StringIO.StringIO()
  yar = YAR(yar_file, mode='w')
  AddAdditionalTestFiles(yar)
  yar.Write(reg_entries_file, 'yawt_reg.txt')
  yar.Write(file_entries_file, 'yawt_file.txt')
  return yar_file.getvalue()


########################################################################
#
# Utility Functions
#
########################################################################

def LoadInstallData(yar_file):
  """Read and parse the installation data from a YAWT test.

  Args:
    yar_file: the name of the YAR file containing the YAWT test

  Returns:
    a tuple containing:
      reg_data: a WinRegParser instance
      file_data: a dictionary of file names and their sha1sums
      reg_todo: a dictionary of registry entries marked as 'todo'
      file_todo: a dictionary of file entries marked as 'todo'
  """
  yar = YAR(yar_file)
  reg_content = yar.Read('yawt_reg.txt')
  reg_data = WinRegParser(reg_content)
  file_content = yar.Read('yawt_file.txt')
  file_data = LineSeparatedKeyValueParser(file_content)
  reg_todo_content = yar.Read('yawt_reg_todo.txt')
  reg_todo = RegTodoEntriesParser(reg_todo_content)
  file_todo_content = yar.Read('yawt_file_todo.txt')
  file_todo = LineSeparatedDataParser(file_todo_content)
  yar.Close()
  return (reg_data, file_data, reg_todo, file_todo)


def VerifyType(param, type_obj, size=0):
  """Verify some object matches a certain type, and optionally verify the
  object size.

  For some object, make sure it has the right type, and if appropriate, check
  to see if it has a certain size.

  Args:
    param: the object to check
    type_obj: a Type object to check against
    size: the size of an object
  """

  if type(param) != type_obj:
    raise WineTestTypeError('expecting %s, got %s' % (type_obj, type(param)))

  if type_obj == TupleType and size > 0 and len(param) != size:
    raise WineTestConfigError('expecting tuple of size %d, got %d' % (size,
      len(param)))


def IsSha1sum(sum):
  """Check to see if a string is a valid sha1sum.

  Args:
    sum: the sha1 checksum string

  Returns:
    if the string is a valid sha1sum: True
    if the string has the wrong length or invalid characters: False
  """

  if len(sum) != 40:
    return False
  if re.search('[^0-9a-f]', sum):
    return False
  return True


def VerifySha1sum(file, checksum):
  """Use sha1sum to check the file and return whether the checksum match.

  Args:
    checksum: the sha1sum to check against

  Returns:
    if the sha1sum matches: True
    if the sha1sum dose not match or we cannot read the file: False
  """


  # get the executable
  try:
    executable = open(file, 'rb')
    data = executable.read()
    executable.close()
  except IOError:
    return False

  # sha1sum it
  sha1summer = sha.new(data)
  sha1sum = sha1summer.hexdigest()

  executable.close()

  return checksum == sha1sum


def VerifyInstallData(reg_entries, file_entries):
  """Verify the installation data is valid.

  Make sure the registry and file information in the installation data are
  valid.

  Args:
    reg_entries: { (key, name): (value, type, todo), ... }
    file_entries: { filename: (sha1sum, todo), ... }
  """

  hkey_prog = re.compile('^HKEY_')
  for k, v in reg_entries.iteritems():
    VerifyType(k, TupleType, size=2)
    VerifyType(k[0], StringType)
    VerifyType(k[1], StringType)
    VerifyType(v, TupleType, size=3)
    VerifyType(v[0], StringType)
    VerifyType(v[1], StringType)
    VerifyType(v[2], IntType)
    if hkey_prog.search(k[0]) == None:
      raise WineTestConfigError

  for k, v in file_entries.iteritems():
    VerifyType(k, StringType)
    VerifyType(v, TupleType, size=2)
    VerifyType(v[0], StringType)
    VerifyType(v[1], IntType)
    if not IsSha1sum(v[0]):
      raise WineTestConfigError


def ConfigureLogging(verbosity):
  """Set the logger's verbosity level.

  Args:
    verbosity: 1 (least verbose) - 5 (most verbose)
  """

  VerifyType(verbosity, IntType)
  if verbosity < 1:
    verbosity = 1
  elif verbosity > 5:
    verbosity = 5

  # normalize the verbosity level to what the python logger uses
  verbosity = 10 * (6 - verbosity)
  logging.basicConfig(format='%(message)s')
  root_logger = logging.getLogger()
  root_logger.level = verbosity


def GetCaseInsensitiveFilename(drive_prefix, filename):
  """Use case insensitive matching to find a given file.

  Given a unix filename, look for it under drive_prefix, and return
  drive_prefix concatenated with the 'best match' for the filename. To get the
  best match, first try a case sensitive match, and if that fails, try a case
  insensitive match. When two or more files matches the name, but different
  cases, then the result of the returned result is undefined. If the file
  does not exist for the case insensitive search, then we return the original
  filename.

  Args:
    drive_prefix: Path to a directory that serves as the root directory for a
      windows drive
    filename: Relative path to a file

  Returns:
    if there exists a match: the case insensitive path to the file
    if there exists multiple matches: the first match
    if no match exists: return the original filename
  """

  # try to see if we can find the file as it is
  filename = os.path.normpath(filename)
  fullname = os.path.join(drive_prefix, filename)
  if os.path.exists(fullname):
    return fullname

  # if not then we first split the filename
  tmp_filename = filename
  split_filename = []
  while True:
    if tmp_filename == "" or tmp_filename == '/' or tmp_filename == '\\':
      break
    s = os.path.split(tmp_filename)
    split_filename.insert(0, s[1].lower())
    tmp_filename = s[0]

  # starting from the drive_prefix
  possible_paths = [drive_prefix]

  # we do a BFS through the directory hierarchy
  for name in split_filename:
    new_possible_paths = []
    for path in possible_paths:
      files_in_path = os.listdir(path)
      for f in files_in_path:
        if name == f.lower():
          new_possible_paths.append(os.path.join(path, f))
    possible_paths = new_possible_paths

  if possible_paths == []:
    return fullname
  return possible_paths[0]


def DownloadFile(url, filename):
  """Download url and save it as filename.

  Args:
    url: the resource to download
    filename: name of the file to save the download as

  Returns:
    True if the download succeeds
  """

  # get the file
  logging.info("Downloading %s" % url)
  try:
    f = open(filename, 'wb')
    download = urllib.urlopen(url)
    f.write(download.read())
    f.close()
  except IOError, (error, strerror):
    logging.error("DownloadFile I/O error %s - %s: %s" % (error, filename,
      strerror))
    return False

  logging.info("Fetched %s" % url)
  return True


def IsBinary(data):
  """Determine if a string is UTF-8 text or binary data.

  Args:
    data: some arbitrary string

  Returns:
    True if data contains non-printable characters
  """
  ascii_printable = (string.ascii_letters + string.digits + string.punctuation
      + string.whitespace)

  if not isinstance(data, basestring):
    raise TypeError

  try:
    unicode_string = data.decode('utf8')
  except UnicodeDecodeError:
    # if we cannot decode as utf8, then it can't be ascii or valid unicode
    return True

  for c in unicode_string:
    unichar_code = ord(c)
    if unichar_code < 256:
      if unichar_code >= 128:
        c = chr(unichar_code - 128)
      if c not in ascii_printable:
        return True
  return False


def IsFileLikeObject(obj):
  """Determine if obj is a file-like object.

  Args:
    obj: any object

  Returns:
    True if obj has the methods a file-like object should have
  """
  file_methods = ['close', 'flush', 'read', 'readline', 'readlines', 'seek',
    'tell', 'truncate', 'write', 'writelines']

  if not isinstance(obj, object):
    return False

  # get a list of (method name, method object) tuples
  members = inspect.getmembers(obj)
  for m in members:
    (name, method) = m
    if name in file_methods:
      if not inspect.isroutine(method):
        return False
      file_methods.remove(name)

  # if anything in file_methods did not get removed in the previous loop
  if len(file_methods):
    return False
  return True


def ExtractYAR(yar, file_list, base_dir, overwrite=False):
  """Extracts files from a YAR object.

  Args:
    yar: the YAR object to extract from
    file_list: list of files to extract from yar.List()
    base_dir: the directory to extract into
    overwrite: overwrites existing files if set to True

  Returns:
  """

  return_value = 0
  for (file_name, file_type) in file_list:
    full_file_path = os.path.join(base_dir, file_name)
    if os.path.exists(full_file_path):
      # don't recreate directories that already exists
      if file_list == 'directory':
        continue

      # check to see if we want to / can overwrite an existing file
      if not overwrite:
        logging.info("%s already exists, skipping" % file_name)
        return_value = 1
        continue
      if not os.access(full_file_path, os.W_OK):
        logging.error("cannot overwrite %s, skipping" % file_name)
        return_value = 1
        continue
    else:
      # check to see if the file's parent directory exists
      parent_dir = os.path.dirname(full_file_path)
      if parent_dir and not os.path.exists(parent_dir):
        try:
          os.makedirs(parent_dir)
        except IOError, (error, strerror):
          logging.error("%s - %s: %s" % (error, full_file_path, strerror))
          return_value = 1
          continue

    # create the directory
    if file_type == 'directory':
      try:
        os.makedirs(full_file_path)
      except IOError, (error, strerror):
        logging.error("%s - %s: %s" % (error, full_file_path, strerror))
        return 1
      continue

    # create the file
    try:
      # read the file data
      data = yar.Read(file_name)
      if data == None:
        logging.error("YAR file does not contain %s")
        return_value = 1
      # write it out
      file_handle = open(full_file_path, 'wb')
      file_handle.write(data)
    except IOError, (error, strerror):
      logging.error("%s - %s: %s" % (error, full_file_path, strerror))
      return_value = 1
  return return_value



########################################################################
#
# YAWT
#
########################################################################

def PrintVersion(program_name):
  """Print the program version number.

  Args:
    program_name: path to the executable
  """
  print "%s version 0.0.0" % os.path.basename(program_name)


def PrintUsage(program_name, longhelp=False):
  """Print the usage information.

  Args:
    program_name: path to the executable
    longhelp:     prints all the available commands if set to True
  """
  PrintVersion(program_name)

  print ("YAWT usage: %s [option]... command [app]..." %
      os.path.basename(program_name))

  print """
  Commands:
    get-tools           Download the extra tools required to run YAWT
    get-installers      Download the installables
    install             Install the applications

  Common Options:
    -q, --quiet         Do not display any output
    -v, --verbose       Display debugging output
    -h, --help          Print this message and exit
        --longhelp      Print all available commands and exit
    -V, --version       Print version number and exit

  Get-Tools Options:
    -d, --dry-run       Do not perform the downloads
    -o, --overwrite     Overwrite existing files

  Fetch Options:
    -d, --dry-run       Do not perform the downloads
    -o, --overwrite     Overwrite existing files

  Install Options:
    -d, --dry-run       Do not perform the installs
    -n, --no-verify     Do not verify the installs

  """

  if not longhelp:
    return

  print ("YAR usage: %s [option]... command [arg]..." %
      os.path.basename(program_name))

  print """
  Commands:
    create              Create a Yet another ARchive
    extract             Extract a Yet another ARchive
    list                List the files in a Yet another ARchive
    c                   Same as create
    x                   Same as extract
    l                   Same as list

  Common Options:
    -q, --quiet         Do not display any output
    -v, --verbose       Display debugging output
    -h, --help          Print this message and exit
    -V, --version       Print version number and exit

  Extract Options:
    -o, --overwrite     Overwrite existing files

  """

  print ("YAWT LogParser usage: %s [option]... command [infile]..." %
    os.path.basename(program_name))

  print """
  Commands:
    logparse            Parse an install log
    logmerge            Merge multiple logparse outputs

  Common Options:
    -q, --quiet         Do not display any output
    -v, --verbose       Display debugging output
    -h, --help          Print this message and exit
    -V, --version       Print version number and exit
  """

def ParseOptions(argv):
  """Parse the command line.

  Parse the command line options and arguments and display the usage
  information if requested / on error.

  Args:
    argv: list of arguments from the command line

  Returns:
    options: dictionary with all the options
    arguments: list of arguments
  """

  options = {'verbosity_modifier': 0}

  valid_options = {
    'common': ['verbosity_modifier'],
    'get-tools': ['dry_run', 'overwrite'],
    'get-installers': ['dry_run', 'overwrite'],
    'install': ['dry_run', 'no_verify'],
    'create': [],
    'extract': ['overwrite'],
    'list': [],
    'logparse': [],
    'logmerge': [],
  }

  command_aliases = {
    'c': 'create',
    'x': 'extract',
    'l': 'list',
  }

  # min/max number of arguments, excluding argv[0], options, and the command
  # negative numbers mean no min/max
  num_arguments_min_max = {
    'get-tools': (0, 0),
    'get-installers': (0, -1),
    'install': (0, -1),
    'create': (2, -1),
    'extract': (1, -1),
    'list': (1, -1),
    'logparse': (1, 1),
    'logmerge': (2, -1),
  }

  try:
    opts, arguments = getopt.getopt(argv[1:], 'dLhoqvV', ['dry-run', 'help',
        'longhelp', 'overwrite', 'quiet', 'verbose', 'version'])
  except getopt.GetoptError:
    # print help information and exit:
    PrintUsage(argv[0])
    sys.exit(1)
  except:
    logging.error('Unexpected error: %s' % sys.exc_info()[0])
    raise

  for o, a in opts:
    if o in ('-d', '--dry-run'):
      options['dry_run'] = 1
    if o in ('-o', '--overwrite'):
      options['overwrite'] = 1
    if o in ('-q', '--quiet'):
      options['verbosity_modifier'] -= 1
    if o in ('-v', '--verbose'):
      options['verbosity_modifier'] += 1
    if o in ('-h', '--help'):
      PrintUsage(argv[0])
      sys.exit(0)
    if o == '--longhelp':
      PrintUsage(argv[0], longhelp=True)
      sys.exit(0)
    if o in ('-V', '--version'):
      PrintVersion(argv[0])
      sys.exit(0)

  # get the command and check for invalid options
  if arguments:
    # check to see if the command was an alias
    if arguments[0] in command_aliases:
      arguments[0] = command_aliases[arguments[0]]
    if arguments[0] in valid_options:
      for o in options:
        if o in valid_options['common']:
          continue
        if o not in valid_options[arguments[0]]:
          logging.error('%s is not a valid option for the %s command' %
              (o, arguments[0]))
          sys.exit(1)
      # no invalid options
      num_arguments = len(arguments) - 1
      (min_args, max_args) = num_arguments_min_max[arguments[0]]
      if num_arguments < min_args:
        logging.error('%s command requires at least %d arguments' %
            (arguments[0], min_args))
        sys.exit(1)
      if max_args >= 0 and num_arguments > max_args:
        logging.error('%s command takes at most %d arguments' % (arguments[0],
          max_args))
        sys.exit(1)
      # number of arguments is within range, return options and arguments
      return options, arguments

  # no arguments or invalid command
  PrintUsage(argv[0])
  sys.exit(1)


def DoGetTools(options, program_path):
  """Download the extra tools required to run YAWT.

  Args:
    options: dictionary of options
    program_path: the path to this script

  Returns:
    on success: 0
    on failure: 1 or greater
  """


  try:
    conf = GetToolsConfig(os.path.dirname(program_path), options)
  except WineTestConfigError, detail:
    logging.error('Error while initializing configuration: %s', detail)
    sys.exit(1)
  ConfigureLogging(conf.Get('verbosity'))

  tool_fetcher = ToolFetcher(conf)
  if not tool_fetcher.FetchTools():
    sys.exit(1)


def DoFetch(options, arguments, program_path):
  """Download the applications.

  Download applications listed in arguments, or all the applications if no
  application is specified.

  Args:
    options: dictionary of options
    arguments: list of programs to install, can be empty
    program_path: the path to this script

  Returns:
    on success: 0
    on failure: 1 or greater
  """


  try:
    conf = FetchConfig(os.path.dirname(program_path), options)
  except WineTestConfigError, detail:
    logging.error('Error while initializing configuration: %s', detail)
    sys.exit(1)
  ConfigureLogging(conf.Get('verbosity'))

  if arguments:
    app_list = arguments
  else:
    app_list = glob.glob('%s/*/*.yar' % conf.Get('test_dir'))

  for p in app_list:
    try:
      profile = AppData(p)
    except (WineTestTypeError, WineTestConfigError):
      logging.error("Invalid application profile in %s" % p)
      continue
    except YARError:
      logging.error("%s contains errors" % p)
      continue
    except IOError, (error, strerror):
      logging.error("I/O error %s - %s: %s" % (error, p, strerror))
      continue

    fetcher = InstallableFetcher(profile, conf)
    fetcher.FetchInstallable()


def DoInstall(options, arguments, program_path):
  """Install the applications.

  Install applications listed in arguments, or all the applications if no
  application is specified.


  Args:
    options: dictionary of options
    arguments: list of programs to install, can be empty
    program_path: the path to this script

  Returns:
    on success: 0
    on failure: 1 or greater
  """

  try:
    conf = InstallConfig(os.path.dirname(program_path), options)
  except WineTestConfigError, detail:
    logging.error('Error while initializing configuration: %s', detail)
    sys.exit(1)

  ConfigureLogging(conf.Get('verbosity'))
  good_apps = 0
  bad_apps = 0

  if arguments:
    app_list = arguments
  else:
    app_list = glob.glob('%s/*/*.yar' % conf.Get('test_dir'))

  for p in app_list:
    try:
      app = Installer(conf, p)
    except (WineTestTypeError, WineTestConfigError):
      logging.error("test data in %s contains errors" % p)
      continue
    except YARError:
      logging.error("%s contains errors" % p)
      continue
    except IOError, (error, strerror):
      logging.error("I/O error %s - %s: %s" % (error, p, strerror))
      continue

    install_result = app.InstallApplication()
    if install_result:
      good_apps += 1
    else:
      bad_apps += 1

  if bad_apps == 0:
    logging.info("PASS: All installations succeeded! (%d)" % good_apps)
    exit_code = 0
  else:
    logging.info("FAIL: %d application install(s) succeeded, "
        "%d application installs failed." % (good_apps, bad_apps))
    exit_code = 1

  sys.exit(exit_code)


def DoYARCreate(options, yar_name, filenames):
  """Create a yar archive.

  Args:
    options: dictionary of optional parameters for creating files
    yar_name: name of the YAR file
    filenames: names of the files to put into the YAR file

  Returns:
    on success: 0
    if any error occurs: 1 or greater
  """
  # change how much data we should output
  ConfigureLogging(4 + options['verbosity_modifier'])

  if not yar_name:
    logging.error("no YAR file specified")
    sys.exit(1)
  if len(filenames) < 1:
    logging.error("bravely refusing to create an empty archive")
    sys.exit(1)

  exit_value = 0
  yar = YAR(yar_name, mode='w')
  for f in filenames:
    try:
      file_handle = open(f, 'rb')
    except IOError, (error, strerror):
      # log and skip files we cannot open
      logging.error("%s - %s: %s" % (error, f, strerror))
      exit_value = 1
      continue
    try:
      yar.Write(file_handle, f)
    except IOError, (error, strerror):
      logging.error("%s - %s: %s" % (error, f, strerror))
      sys.exit(1)
    except YARError, error_msg:
      logging.error("YARError: %s" % error_msg)
    file_handle.close()
  sys.exit(exit_value)


def DoYARExtract(options, yar_name, files_to_extract):
  """Extract the files in the archives.

  Like tar, we'll ignore directories that already exist, but unlike tar, we
  will not overwrite files unless explicitly told to.

  Args:
    options: dictionary of optional parameters for listing files
    yar_name: name of the YAR file to extract files from
    files_to_extract: names of the files to extract from the YAR file

  Returns:
    on success: 0
    if any error occurs: 1 or greater
  """
  # change how much data we should output
  ConfigureLogging(4 + options['verbosity_modifier'])

  exit_value = 0
  # open and get the file list
  try:
    yar = YAR(yar_name)
    file_list = yar.List()
  except IOError, (error, strerror):
    logging.error("%s - %s: %s" % (error, yar_name, strerror))
    sys.exit(1)
  except YARError, error_msg:
    logging.error("YARError: %s" % error_msg)
    sys.exit(1)

  new_file_list = []
  for f in files_to_extract:
    found = False
    for (file_name, file_type) in file_list:
      if f == file_name:
        found = True
        break
    if found:
      new_file_list.append((file_name, file_type))
    else:
      logging.error("%s is not in the archive" % f)
      exit_value = 1
  if files_to_extract:
    file_list = new_file_list

  if 'overwrite' in options:
    overwrite = options['overwrite']
  else:
    overwrite = False

  r = ExtractYAR(yar, file_list, '', overwrite=overwrite)
  if r > 0:
    exit_value = r
  yar.Close()
  sys.exit(exit_value)


def DoYARList(options, yar_name, files_to_list):
  """List the files in the archives.

  Args:
    options: dictionary of optional parameters for listing files
    yar_name: name of the YAR file to list files from
    files_to_list: names of the files to list from the YAR file

  Returns:
    on success: 0
    on failure: 1 or greater
  """

  # change how much data we should output
  ConfigureLogging(4 + options['verbosity_modifier'])

  try:
    yar = YAR(yar_name)
    file_list = yar.List()
    yar.Close()
  except IOError, (error, strerror):
    logging.error("%s - %s: %s" % (error, yar_name, strerror))
    sys.exit(1)
  except YARError, error_msg:
    logging.error("YARError: %s" % error_msg)
    sys.exit(1)

  exit_value = 0
  new_file_list = []
  for f in files_to_list:
    found = False
    for (file_name, file_type) in file_list:
      if f == file_name:
        found = True
        break
    if found:
      new_file_list.append((file_name, file_type))
    else:
      logging.error("%s is not in the archive" % f)
      exit_value = 1
  if files_to_list:
    file_list = new_file_list

  for (file_name, file_type) in file_list:
    print file_name

  sys.exit(exit_value)


def DoLogParse(options, infile):
  """Parse an InCtrl5 install log and write the installation data to stdout.

  Take an InCtrl5 install log in plan text format, parse it, checksum the newly
  installed files, and output the installation data to stdout.

  Args:
    options: dictionary of optional parameters for listing files
    infile: the name of the InCtrl5 log file

  Returns:
    on success: 0
    on failure: 1 or greater
  """

  # change how much data we should output
  ConfigureLogging(4 + options['verbosity_modifier'])

  # Read the data
  try:
    in_handle = open(infile, 'r')
    in_data = in_handle.read()
    in_data = in_data.decode(locale.getdefaultlocale()[1]).encode('utf8')
  except IOError, (error, strerror):
    logging.error("%s - %s: %s" % (error, infile, strerror))
    sys.exit(1)

  lines = in_data.splitlines()
  if lines[1] != "Generated by InCtrl5, version 1.0.0.0":
    logging.error('Not an InCtrl5 log file')
    sys.exit(1)

  reg_entries = ParseInctrlLogRegEntries(lines)
  file_entries = ParseInctrlLogFileEntries(lines)

  print GenerateYAWTTest(reg_entries, file_entries)
  sys.exit(0)


def DoLogMerge(options, infiles):
  """Merge logparse outputs together and write the installation data to stdout.

  Merge multiple installation data files together into a single installation
  file that accounts for inconsistencies between repeated installs.

  Args:
    options: dictionary of optional parameters for listing files
    infiles: the name of the parsed InCtrl5 logs

  Returns:
    on success: 0
    on failure: 1 or greater
  """

  # change how much data we should output
  ConfigureLogging(4 + options['verbosity_modifier'])

  (reg1, file1, reg1_todo, file1_todo) = LoadInstallData(infiles[0])
  del (reg1_todo, file1_todo)

  for f in infiles[1:]:
    (reg2, file2, reg2_todo, file2_todo) = LoadInstallData(f)
    del (reg2_todo, file2_todo)

    keys_in_common = FindCommonDictionary(file1, file2)
    # remove files not in common
    new_file = {}
    for k in file1.iterkeys():
      if k in keys_in_common:
        new_file[k] = file1[k]
    file1 = new_file

    # remove registry sections not in common
    new_reg = copy.deepcopy(reg1)
    for s in reg1.iterkeys():
      if s not in reg2:
        del new_reg[s]
      else:
        k1 = reg1[s]
        k2 = reg2[s]
        keys_in_common = FindCommonDictionary(k1, k2)
        for k in k1.iterkeys():
          if k not in keys_in_common:
            del new_reg[s][k]
    reg1 = new_reg

  print GenerateYAWTTest(reg1, file1)
  sys.exit(0)


def main(argv):
  """Get the options and arguments and run the appropriate command.

  Args:
    argv: list of arguments and options from the command line
  """
  # initialize logging first
  ConfigureLogging(4)
  command_line_options, arguments = ParseOptions(argv)

  if arguments[0] == 'get-installers':
    DoFetch(command_line_options, arguments[1:], argv[0])
  elif arguments[0] == 'install':
    DoInstall(command_line_options, arguments[1:], argv[0])
  elif arguments[0] == 'get-tools':
    DoGetTools(command_line_options, argv[0])
  elif arguments[0] == 'create':
    DoYARCreate(command_line_options, arguments[1], arguments[2:])
  elif arguments[0] == 'extract':
    DoYARExtract(command_line_options, arguments[1], arguments[2:])
  elif arguments[0] == 'list':
    DoYARList(command_line_options, arguments[1], arguments[2:])
  elif arguments[0] == 'logparse':
    DoLogParse(command_line_options, arguments[1])
  elif arguments[0] == 'logmerge':
    DoLogMerge(command_line_options, arguments[1:])


if __name__ == '__main__':
  main(sys.argv)
